---
title: "APSR Additional Study Details"
author: "Velez, Liu"
output:
  html_document: default
  pdf_document: default
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(warning = FALSE, echo = FALSE, message = FALSE)
```

# Additional Study Details

## Additional Study Details A.1 (Full Model Results, Exp 1)
```{r}
source("scripts/clean_study1.R")

modelsummary:::modelsummary(list(lm_robust(post_attitudes ~ motivation*information + 
                                             scale(pre_attitudes) + scale(pre_certainty), study1),
                                 lm_robust(post_certainty ~ motivation*information +  
                                             scale(pre_attitudes) + scale(pre_certainty), study1)))
```

## Additional Study Details A.2 (Full Model Results, Exp 2)

```{r}
source("scripts/clean_study2.R")

modelsummary:::modelsummary(list(lm_robust(post_attitudes ~ motivation*information*condition + 
                              scale(pre_attitudes) + scale(pre_certainty), combined_df),
                                 lm_robust(post_certainty ~ motivation*information*condition +  
                               scale(pre_attitudes) + scale(pre_certainty), combined_df),
                            lm_robust(post_certain_scores ~ motivation*information*condition +  
                              scale(pre_attitudes) + scale(pre_certain_scores), combined_df)))
```

## Additional Study Details A.3 (Full Model Results, Exp 3)

```{r}
source("scripts/clean_study3.R")

modelsummary:::modelsummary(list(lm_robust(post_outcome_1 ~ treatment, study3),
                                 lm_robust(post_outcome_1 ~ treatment + scale(age) + 
                                             scale(education) + scale(hhi)
            + scale(ideology) + scale(white) + scale(political_party), study3),
                            lm_robust(post_certain_scores ~ treatment, study3),
            lm_robust(post_certain_scores ~ treatment + scale(age) + 
                                             scale(education) + scale(hhi)
            + scale(ideology) + scale(white) + scale(political_party), study3)), stars=TRUE)
```

## Additional Study Details A.4 (Full Model Results, Exp 4)

```{r}
source("scripts/clean_study4.R")

model1 <- lm_robust(pw2p_scale ~ treatment + likert + certainty +
                      age + educ + income + ideology + participation_scale +
                      pk_scale + pid7, study4_coded,
                    subset = is.na(generic_flag))

model1b <- lm_robust(pw2p_scale ~ treatment, study4_coded,
                     subset = is.na(generic_flag))


model2 <- lm_robust(pw2p_defend ~ treatment + likert + certainty +
                      age + educ + income + ideology + participation_scale + pk_scale +
                      pid7, study4_coded, subset = is.na(generic_flag))

model2b <- lm_robust(pw2p_defend ~ treatment,
                     study4_coded, subset = is.na(generic_flag))

modelsummary:::modelsummary(list(model1, model1b, model2, model2b), 
                            fmt = 2, stars = TRUE)
```

## Additional Study Details A.5 (Full Model Results, Exp 5)

```{r}
source("scripts/clean_study5.R")

model1 <- lm_robust(pw2p_scale ~ as.factor(treatment) + likert + certainty +
                      age + educ + income + ideology + participation_scale +
                      pk_scale + pid7, study5_coded,
                    subset = is.na(generic_flag))

model1b <- lm_robust(pw2p_scale ~ as.factor(treatment), study5_coded,
                     subset = is.na(generic_flag))


model2 <- lm_robust(pw2p_defend ~ as.factor(treatment) + likert + certainty +
                      age + educ + income + ideology + participation_scale + pk_scale +
                      pid7, study5_coded, subset = is.na(generic_flag))

model2b <- lm_robust(pw2p_defend ~ as.factor(treatment),
                     study5_coded, subset = is.na(generic_flag))

modelsummary:::modelsummary(list(model1, model1b, model2, model2b),
                            fmt = 2, stars = TRUE)
```

## Additional Study Details A.6 (Full Model Results, Prior Attitude Effect)
```{r}
source("scripts/clean_study2.R")

combined_df %>%
 group_by(condition) %>%
 do(mod = broom::tidy(emmeans(lm_robust(post_con_strength ~ motivation + 
                                         pre_attitudes, .,
                                        clusters = participantId),
                              specs = pairwise ~ motivation)$emmeans)) %>%
 unnest(c(mod)) %>%
 bind_cols(Outcome = "Strength",
           Type = "Con Args") %>%
 bind_rows(combined_df %>%
            group_by(condition) %>%
            do(mod = broom::tidy(emmeans(lm_robust(post_pro_strength ~ motivation + 
                                                    pre_attitudes, .,
                                                   clusters = participantId),
                                         specs = pairwise ~ motivation)$emmeans)) %>%
            unnest(c(mod)) %>%
            bind_cols(Outcome = "Strength",
                      Type = "Pro Args")) %>%
 bind_rows(combined_df %>%
            group_by(condition) %>%
            do(mod = broom::tidy(emmeans(lm_robust(post_con_acc ~ motivation + 
                                                    pre_attitudes, .,
                                                   clusters = participantId),
                                         specs = pairwise ~ motivation)$emmeans)) %>%
            unnest(c(mod)) %>%
            bind_cols(Outcome = "Accuracy",
                      Type = "Con Args")) %>%
 bind_rows(combined_df %>%
            group_by(condition) %>%
            do(mod = broom::tidy(emmeans(lm_robust(post_pro_acc ~ motivation + 
                                                    pre_attitudes, .,
                                                   clusters = participantId),
                                         specs = pairwise ~ motivation)$emmeans)) %>%
            unnest(c(mod)) %>%
            bind_cols(Outcome = "Accuracy",
                      Type = "Pro Args")) %>%
 mutate(`Issue Strength` = plyr::mapvalues(condition,
                                           c("strong", "weak"),
                                           c("Core",
                                             "Peripheral")),
        Motivation = plyr::mapvalues(motivation,
                                     c("accuracy", "directional"),
                                     c("Accuracy Prime",
                                       "Directional Prime"))) %>% 
 select(Outcome, Type, `Issue Strength`, Motivation, estimate, std.error, df, p.value)
```

## Additional Study Details A.7 (Full Model Results, Disconfirmation Bias)
```{r}
source("scripts/clean_study2.R")

modelsummary:::modelsummary(list(lm_robust(share_den ~ 
                                            information*condition, combined_df),
                                 lm_robust(log(timer) ~ 
                                            information*condition, combined_df)))
```

## Additional Study Details B.1 (Change Score Analysis, Exp 2)

```{r}
source("scripts/clean_study2.R")

# ASD Figure B1 Attitude Change Figure
combined_df %>%
  group_by(`Issue Strength`) %>%
  do(mod = broom::tidy(emmeans(lm_robust(I(post_attitudes - pre_attitudes) ~
                                           information, .,
                                         clusters = participantId),
                               specs = pairwise ~ information)$emmeans)) %>%
  unnest(c(mod)) %>%
  ggplot(aes(x = information, y = estimate,
             group = `Issue Strength`)) +
  geom_point(position = position_dodge(.5)) +
  geom_point(position = position_dodge(.5), size = 4) +
  geom_linerange(aes(ymin = estimate - 1.405*std.error,
                     ymax = estimate + 1.405*std.error),
                 position = position_dodge(.5), size = 2) +
  geom_linerange(aes(ymin = estimate - 1.96*std.error,
                     ymax = estimate + 1.96*std.error),
                 position = position_dodge(.5)) +
  theme_ipsum_rc() +
  facet_wrap(. ~ `Issue Strength`, ncol = 2) +
  labs(y = "Estimate") +
  theme(panel.background = element_rect(fill = 'gray95'),
        panel.grid.minor = element_blank(),
        panel.grid.major = element_blank(),
        axis.text.x = element_text(size = 15),
        axis.text.y = element_text(size = 15),
        axis.title.y = element_text(size = 15),
        axis.title.x = element_text(size = 15),
        legend.position = "bottom") +
  labs(x = "", y = "\u0394 Attitude Strength") +
  geom_hline(yintercept = 0, linetype = "dotted") 
```

```{r}
# ASD Figure B1 Certainty Change
combined_df %>%
  group_by(condition) %>%
  do(mod = broom::tidy(emmeans(lm_robust(I(post_certain_scores - pre_certain_scores) ~ information, .,
                                         clusters = participantId),
                               specs = pairwise ~ information)$emmeans)) %>%
  unnest(c(mod)) %>%
  mutate(information = plyr::mapvalues(information,
                                       c("con", "mixed", "pro"),
                                       c("Cons",
                                         "Mixed",
                                         "Pros")),
         `Issue Strength` = plyr::mapvalues(condition,
                                            c("strong", "weak"),
                                            c("Strong",
                                              "Weak"))) %>%
  ggplot(aes(x = information, y = estimate,
             group = `Issue Strength`)) +
  geom_point(position = position_dodge(.5)) +
  geom_point(position = position_dodge(.5), size = 4) +
  geom_linerange(aes(ymin = estimate - 1.405*std.error,
                     ymax = estimate + 1.405*std.error),
                 position = position_dodge(.5), size = 2) +
  geom_linerange(aes(ymin = estimate - 1.96*std.error,
                     ymax = estimate + 1.96*std.error),
                 position = position_dodge(.5)) +
  theme_ipsum_rc() +
  facet_wrap(. ~ `Issue Strength`, scales = "free",
             ncol = 2) +
  labs(y = "Estimate") +
  theme(panel.background = element_rect(fill = 'gray95'),
        panel.grid.minor = element_blank(),
        panel.grid.major = element_blank(),
        axis.text.x = element_text(size = 15),
        axis.text.y = element_text(size = 15),
        axis.title.y = element_text(size = 15),
        axis.title.x = element_text(size = 15),
        legend.position = "bottom") +
  labs(x = "", y = "\u0394 Attitude Certainty") +
  geom_hline(yintercept = 0, linetype = "dotted") 
```

```{r}
## Model Results
combined_df %>%
  group_by(`Issue Strength`) %>%
  do(mod = broom::tidy(emmeans(lm_robust(I(post_attitudes - pre_attitudes) ~
                                           information, .,
                                         clusters = participantId),
                               specs = pairwise ~ information)$emmeans)) %>% 
 unnest(c(mod)) %>% 
 bind_cols(Outcome = "Δ Strength") %>% 
 bind_rows(combined_df %>%
            group_by(`Issue Strength`) %>%
            do(mod = broom::tidy(emmeans(lm_robust(I(post_certain_scores - 
                                                      pre_certain_scores) ~ information, .,
                                                   clusters = participantId),
                                         specs = pairwise ~ information)$emmeans)) %>%
            unnest(c(mod)) %>% 
            bind_cols(Outcome = "Δ Certainty")) %>% 
 select(Outcome, `Issue Strength`, information, estimate, std.error, df, p.value)
```

```{r}
# ASD Figure B2 Attitude Change Figure
combined_df %>%
  group_by(motivation, `Issue Strength`) %>%
  do(mod = broom::tidy(emmeans(lm_robust(I(post_attitudes - pre_attitudes) ~
                                           information, .,
                                         clusters = participantId),
                               specs = pairwise ~ information)$emmeans)) %>%
  unnest(c(mod)) %>%
  ggplot(aes(x = information, y = estimate, shape = motivation,
             group = motivation)) +
  geom_point(position = position_dodge(.5)) +
  geom_point(position = position_dodge(.5), size = 4) +
  geom_linerange(aes(ymin = estimate - 1.405*std.error,
                     ymax = estimate + 1.405*std.error),
                 position = position_dodge(.5), size = 2) +
  geom_linerange(aes(ymin = estimate - 1.96*std.error,
                     ymax = estimate + 1.96*std.error),
                 position = position_dodge(.5)) +
  theme_ipsum_rc() +
  facet_wrap(. ~ `Issue Strength`, ncol = 2) +
  labs(y = "Estimate") +
  theme(panel.background = element_rect(fill = 'gray95'),
        panel.grid.minor = element_blank(),
        panel.grid.major = element_blank(),
        axis.text.x = element_text(size = 15),
        axis.text.y = element_text(size = 15),
        axis.title.y = element_text(size = 15),
        axis.title.x = element_text(size = 15),
        legend.position = "bottom") +
  labs(x = "", y = "\u0394 Attitude Strength") +
  geom_hline(yintercept = 0, linetype = "dotted") 
```

```{r}
# ASD Figure B2 Certainty Change
combined_df %>%
  group_by(motivation, condition) %>%
  do(mod = broom::tidy(emmeans(lm_robust(I(post_certain_scores - pre_certain_scores) ~ information, .,
                                         clusters = participantId),
                               specs = pairwise ~ information)$emmeans)) %>%
  unnest(c(mod)) %>%
  mutate(information = plyr::mapvalues(information,
                                       c("con", "mixed", "pro"),
                                       c("Cons",
                                         "Mixed",
                                         "Pros")),
         `Issue Strength` = plyr::mapvalues(condition,
                                            c("strong", "weak"),
                                            c("Strong",
                                              "Weak"))) %>%
  ggplot(aes(x = information, y = estimate,
             group = motivation, shape = motivation)) +
  geom_point(position = position_dodge(.5)) +
  geom_point(position = position_dodge(.5), size = 4) +
  geom_linerange(aes(ymin = estimate - 1.405*std.error,
                     ymax = estimate + 1.405*std.error),
                 position = position_dodge(.5), size = 2) +
  geom_linerange(aes(ymin = estimate - 1.96*std.error,
                     ymax = estimate + 1.96*std.error),
                 position = position_dodge(.5)) +
  theme_ipsum_rc() +
  facet_wrap(. ~ `Issue Strength`, scales = "free",
             ncol = 2) +
  labs(y = "Estimate") +
  theme(panel.background = element_rect(fill = 'gray95'),
        panel.grid.minor = element_blank(),
        panel.grid.major = element_blank(),
        axis.text.x = element_text(size = 15),
        axis.text.y = element_text(size = 15),
        axis.title.y = element_text(size = 15),
        axis.title.x = element_text(size = 15),
        legend.position = "bottom") +
  labs(x = "", y = "\u0394 Attitude Certainty") +
  geom_hline(yintercept = 0, linetype = "dotted") 
```

```{r}
## Model Results
combined_df %>%
  group_by(motivation, `Issue Strength`) %>%
  do(mod = broom::tidy(emmeans(lm_robust(I(post_attitudes - pre_attitudes) ~
                                           information, .,
                                         clusters = participantId),
                               specs = pairwise ~ information)$emmeans)) %>%
  unnest(c(mod)) %>% 
 bind_cols(Outcome = "Δ Strength") %>% 
 bind_rows(combined_df %>% 
            group_by(motivation, `Issue Strength`) %>%
            do(mod = broom::tidy(emmeans(lm_robust(I(post_certain_scores - pre_certain_scores) ~
                                                    information, .,
                                                   clusters = participantId),
                                         specs = pairwise ~ information)$emmeans)) %>%
            unnest(c(mod)) %>% 
            bind_cols(Outcome = "Δ Certainty")) %>% 
 select(Outcome, motivation, `Issue Strength`, information, estimate, std.error, df, p.value)
```

## Additional Study Details B.2 (Are GPT-3 Args Persuasive, Full Results)

```{r}
source("scripts/prep_appendix_B2.R")
library(estimatr)

combined_pairs %>%
  group_by(subtopic) %>%
  do(mod = broom::tidy(lm_robust(rating ~ 1, .,
                                 clusters = participant))) %>%
  unnest(c(mod))
```

## Additional Study Details B.3 (Completion by Issue Topic, Exp 2)

```{r}
source("scripts/prep_appendix_B1.R")
source("scripts/clean_study2.R")

combined_df <- combined_df %>%
 left_join(coded_s2 %>%
            mutate(condition = "strong") %>%
            select(id.x = id, subtopic, majortopic, condition), 
           by = c("id.x", "condition"))

completion <- strong %>%  
 left_join(coded_s2, by = "id") 

completion %>%
 group_by(majortopic) %>% 
 summarize(mean_outcome = mean(generic_flag.x == 1, na.rm = T), n = n(), 
           SE = sd(generic_flag.x)/sqrt(n())) %>%
 na.omit() %>%
 ggplot(aes(x = mean_outcome, y = majortopic)) +
 geom_point(size = 2.5) +
 theme_ipsum_rc() +
 geom_linerange(aes(xmin = mean_outcome - 1.96*SE,
                    xmax = mean_outcome + 1.96*SE),
                size = 0.5) +
 geom_linerange(aes(xmin = mean_outcome - 1.405*SE,
                     xmax = mean_outcome + 1.405*SE),
                 size = 1.2) +
 theme_params +
 theme(axis.text.y = element_text(size = 11),
       axis.text.x = element_text(size = 10),
       axis.title.x = element_text(size = 10)) +
 labs(y = "", x = "GPT-3 Completion Rate by Topic Area")

anova(lm(generic_flag.x ~ majortopic, completion)) %>%
 xtable::xtable()

```

## Additional Study Details B.4 (Certainty Analysis, Exp 3)

```{r}
source("scripts/clean_study3.R")

modelsummary:::modelsummary(list(lm_robust(post_certainty_19 ~ treatment, study3),
                                 lm_robust(post_certainty_19 ~ treatment + scale(age, center = T) + 
                                            scale(education, center = T) + scale(hhi, center = T) + 
                                            scale(ideology, center = T) + scale(white, center = T) +
                                            scale(political_party, center = T), study3)),
                            stars = TRUE)
```

## Additional Study Details B.5 (Robustness of Effects, Exp 3)

```{r}
source("scripts/clean_study3.R")

study3 %<>%
 mutate(pk_ntile = ntile(pk_scale, 3),
        ideology_ntile = ntile(ideology, 3),
        quality_ntile = as.numeric(quality == 1),
        mvc_ntile = as.numeric(mvc_check == 4)) 

pk_cates <- study3 %>%
 group_by(ntile = pk_ntile) %>%
 do(mod = broom::tidy(lm_robust(post_outcome_1 ~ treatment +
                               age + education + hhi +
                               ideology + white + 
                               political_party, data = .))) %>%
 na.omit() %>%
 unnest(c(mod)) %>%
 filter(term == "treatment") %>%
 mutate(variable = "Political Knowledge")

ideo_cates <- study3 %>%
 group_by(ntile = ideology_ntile) %>%
 do(mod = broom::tidy(lm_robust(post_outcome_1 ~ treatment +
                                 age + education + hhi +
                                 white + 
                                 political_party, data = .))) %>%
 na.omit() %>%
 unnest(c(mod)) %>%
 filter(term == "treatment") %>%
 mutate(variable = "Ideology")

quality_cates <- study3 %>%
 group_by(ntile = quality_ntile-1) %>%
 do(mod = broom::tidy(lm_robust(post_outcome_1 ~ treatment +
                                 age + education + hhi +
                                 white + ideology +
                                 political_party, data = .))) %>%
 na.omit() %>%
 unnest(c(mod)) %>%
 filter(term == "treatment") %>%
 mutate(variable = "Response Quality")

attention_cates <- study3 %>%
 group_by(ntile = mvc_ntile-1) %>%
 do(mod = broom::tidy(lm_robust(post_outcome_1 ~ treatment +
                                 age + education + hhi +
                                 white + ideology +
                                 political_party, data = .))) %>%
 na.omit() %>%
 unnest(c(mod)) %>%
 filter(term == "treatment") %>%
 mutate(variable = "Attentiveness")

pk_cates %>%
 bind_rows(ideo_cates) %>%
 bind_rows(quality_cates) %>%
 bind_rows(attention_cates) -> all_cates

all_cates %>%
 mutate(ntile = plyr::mapvalues(ntile,
                                c(-1, 0, 1, 2, 3),
                                c(" Low", "High",
                                  "Tertile 1", "Tertile 2", "Tertile 3"))) %>%
 ggplot(aes(x = ntile, y = estimate)) +
 geom_pointrange(aes(ymin = estimate - 1.96*std.error,
                     ymax = estimate + 1.96*std.error)) +
 geom_pointrange(aes(ymin = estimate - 1.645*std.error,
                     ymax = estimate + 1.645*std.error),
                 size = 1.2) +
 theme_ipsum_rc() +
 theme_params +
 facet_wrap(. ~ variable, scales = "free") +
 geom_hline(yintercept = 0, linetype = "dashed") +
 labs(x = "", y = "Conditional Average Treatment Effect")
```

## Additional Study Details B.6 (Consistency of GPT Output)

```{r}
handcode <- rio::import("data/consistency_handcoded.csv")

vars <- c("likert_doublebarreled", "likert_justif", "likert_empty_unclear", 
          "likert_mischaracterized", "likert_nonattitude_apol_valence",
          "text_pro_flipped", "text_pro_offtopic_nosubstance",
          "text_con_flipped", "text_con_offtopic_nosubstance")

# Table B5 and B6
handcode %>% 
 group_by(study) %>% 
 summarize_at(vars, mean, na.rm = TRUE)

# Table B7
source("scripts/clean_study3.R")
s3_codes <- rio::import("data/study3_aligned_handcoded.csv")

study3 <- study3 %>% 
 left_join(s3_codes, by = "rid")

model <- lm(post_outcome_1 ~ treatment, study3,
   subset = (treatment == 1 & !is.na(true_counter)) | treatment == 0)

stargazer(model, 
        dep.var.labels = "Argument Strength (1-7)",
        covariate.labels = c("Intercept", "Treatment"),
        intercept.bottom = FALSE,
        keep.stat = "n",
        title = "Robustness Check (Experiment 3, Removing Aligned Counterarguments)",
        label = "tab: counter-robustness",
        digits=2)
```

## Additional Study Details B.7 (GPT Formatting Errors)

```{r}
source("scripts/clean_study1.R")
study1$correct_likert <- as.numeric(grepl("^\n\nI believe|^I believe", study1$likert))

source("scripts/clean_study2.R")
combined_df$correct_likert <- as.numeric(grepl("^\n\nI believe|^I believe", combined_df$likert))

s3 <- qualtRics::read_survey("data/study3.csv") %>%
 filter(lubridate::as_date(StartDate) >= "2022-12-12") %>%
 filter(quality > 0 | is.na(quality)) %>%
 filter(is.na(generic_flag)) %>%
 filter(!grepl("farm", generic_flag))

# Table B8
library(modelsummary)
list(lm_robust(correct_likert ~ information + motivation, study1), 
     lm_robust(correct_likert ~ information + motivation, combined_df), 
     lm_robust(correct_likert ~ treatment, study3)) %>% 
 modelsummary(stars = T)
```

```{r}
# Table B9
list(lm_robust(post_attitudes ~ motivation*information*correct_likert + 
                 scale(pre_attitudes) + scale(pre_certainty), study1),
     lm_robust(post_certainty ~ motivation*information*correct_likert +  
                 scale(pre_attitudes) + scale(pre_certainty), study1)) %>% 
  modelsummary(stars = T)
```

```{r}
# Table B10
list(lm_robust(post_attitudes ~ motivation*information*correct_likert + 
                 scale(pre_attitudes) + scale(pre_certainty), 
               combined_df, subset=condition=="strong"),
     lm_robust(post_certain_scores ~ motivation*information*correct_likert +  
                 scale(pre_attitudes) + scale(pre_certainty), 
               combined_df, subset=condition=="strong")) %>% 
  modelsummary(stars = T)
```

```{r}
# Table B11
list(lm_robust(post_outcome_1 ~ treatment*correct_likert + scale(age) + scale(education) + 
                 scale(hhi) + scale(ideology) + scale(white) + 
                 scale(political_party), study3),
     lm_robust(post_certain_scores ~ treatment*correct_likert + scale(age) + scale(education) + 
                 scale(hhi) + scale(ideology) + scale(white) + 
                 scale(political_party), study3)) %>% 
  modelsummary(stars = T)
```

## Additional Study Details B.8 (CATEs by Subtopic)

```{r}
# Experiment 1
source("scripts/clean_study1.R")

subtopics_1 <- read_csv("data/study1_handcoded.csv") %>% 
 transmute(id, subtopic)

for_CATEs_1 <- study1 %>% 
 left_join(subtopics_1, by = "id")


# Estimate CATEs by Subtopic 

# subtopic 1: abortion

list(lm_robust(post_attitudes ~ scale(pre_attitudes) + 
                scale(pre_certainty) + 
                information*motivation, 
               for_CATEs_1, subset=grepl("Abortion", for_CATEs_1$subtopic)),
     lm_robust(post_certainty ~ scale(pre_attitudes) + 
                scale(pre_certainty) + 
                information*motivation, 
               for_CATEs_1, subset=grepl("Abortion", for_CATEs_1$subtopic))) %>%
 modelsummary(stars = T)

# subtopic 2: health care

list(lm_robust(post_attitudes ~ scale(pre_attitudes) + 
                scale(pre_certainty) + 
                information*motivation, 
               for_CATEs_1, subset=grepl("Comprehensive Health Care Reform", for_CATEs_1$subtopic)),
     lm_robust(post_certainty ~ scale(pre_attitudes) + 
                scale(pre_certainty) + 
                information*motivation, 
               for_CATEs_1, subset=grepl("Comprehensive Health Care Reform", for_CATEs_1$subtopic))) %>%
 modelsummary(stars = T)

# subtopic 3: gun control

list(lm_robust(post_attitudes ~ scale(pre_attitudes) + 
                scale(pre_certainty) + 
                information*motivation, 
               for_CATEs_1, subset=grepl("Gun Control", for_CATEs_1$subtopic)),
     lm_robust(post_certainty ~ scale(pre_attitudes) + 
                scale(pre_certainty) + 
                information*motivation, 
               for_CATEs_1, subset=grepl("Gun Control", for_CATEs_1$subtopic))) %>%
 modelsummary(stars = T)
```

```{r}
# Experiment 2
source("scripts/clean_study2.R")
source("scripts/prep_appendix_B1.R")

for_CATEs <- combined_df %>% 
 mutate(id = id.x) %>% 
 left_join(coded_s2, by = "id")

# Estimate CATEs by Subtopic ------------------------------------------------

# subtopic 1: abortion

list(lm_robust(post_attitudes ~ scale(pre_attitudes, center = T) +
                scale(pre_certainty, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Abortion", for_CATEs$subtopic),
               clusters = id),
     lm_robust(post_certainty ~ scale(pre_attitudes, center = T) +
                scale(pre_certainty, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Abortion", for_CATEs$subtopic),
               clusters = id),
     lm_robust(post_certain_scores ~
                scale(pre_attitudes, center = T) +
                scale(pre_certain_scores, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Abortion", for_CATEs$subtopic),
               clusters = id)) %>%
 modelsummary(stars = T)

# subtopic 2: health care

list(lm_robust(post_attitudes ~ scale(pre_attitudes, center = T) +
                scale(pre_certainty, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Comprehensive Health Care Reform", for_CATEs$subtopic),
               clusters = id),
     lm_robust(post_certainty ~ scale(pre_attitudes, center = T) +
                scale(pre_certainty, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Comprehensive Health Care Reform", for_CATEs$subtopic),
               clusters = id),
     lm_robust(post_certain_scores ~
                scale(pre_attitudes, center = T) +
                scale(pre_certain_scores, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Comprehensive Health Care Reform", for_CATEs$subtopic),
               clusters = id)) %>%
 modelsummary(stars = T)

# subtopic 3: gun control

list(lm_robust(post_attitudes ~ scale(pre_attitudes, center = T) +
                scale(pre_certainty, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Gun Control", for_CATEs$subtopic),
               clusters = id),
     lm_robust(post_certainty ~ scale(pre_attitudes, center = T) +
                scale(pre_certainty, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Gun Control", for_CATEs$subtopic),
               clusters = id),
     lm_robust(post_certain_scores ~
                scale(pre_attitudes, center = T) +
                scale(pre_certain_scores, center = T) +
                information*motivation*condition,
               for_CATEs, subset=grepl("Gun Control", for_CATEs$subtopic),
               clusters = id)) %>%
 modelsummary(stars = T)
```

```{r}
# Experiment 3
source("scripts/clean_study3.R")

subtopics_3 <- read_csv("data/study3_handcoded.csv") %>% 
 transmute(rid, subtopic)

for_CATEs_3 <- study3 %>% 
 left_join(subtopics_3, by = "rid")

# Estimate CATEs by Subtopic --------------------------------------

list(lm_robust(post_outcome_1 ~ treatment, 
               for_CATEs_3, subset=grepl("Abortion", for_CATEs_3$subtopic)),
     lm_robust(post_outcome_1 ~ treatment +
                scale(age, center = T) + scale(education, center = T) + scale(hhi, center = T) +
                scale(ideology, center = T) + scale(white, center = T) + 
                political_party, 
               for_CATEs_3, subset=grepl("Abortion", for_CATEs_3$subtopic)),
     lm_robust(post_certain_scores ~ treatment, 
               for_CATEs_3, subset=grepl("Abortion", for_CATEs_3$subtopic)),
     lm_robust(post_certain_scores ~ treatment +
                scale(age, center = T) + scale(education, center = T) + scale(hhi, center = T) +
                scale(ideology, center = T) + scale(white, center = T) + 
                political_party, 
               for_CATEs_3, subset=grepl("Abortion", for_CATEs_3$subtopic))) %>%
 modelsummary(stars = T)


list(lm_robust(post_outcome_1 ~ treatment, 
               for_CATEs_3, subset=grepl("Comprehensive Health Care Reform", for_CATEs_3$subtopic)),
     lm_robust(post_outcome_1 ~ treatment +
                scale(age, center = T) + scale(education, center = T) + scale(hhi, center = T) +
                scale(ideology, center = T) + scale(white, center = T) + 
                political_party, 
               for_CATEs_3, subset=grepl("Comprehensive Health Care Reform", for_CATEs_3$subtopic)),
     lm_robust(post_certain_scores ~ treatment, 
               for_CATEs_3, subset=grepl("Comprehensive Health Care Reform", for_CATEs_3$subtopic)),
     lm_robust(post_certain_scores ~ treatment +
                scale(age, center = T) + scale(education, center = T) + scale(hhi, center = T) +
                scale(ideology, center = T) + scale(white, center = T) + 
                political_party, 
               for_CATEs_3, subset=grepl("Comprehensive Health Care Reform", for_CATEs_3$subtopic))) %>%
 modelsummary(stars = T)


list(lm_robust(post_outcome_1 ~ treatment, 
               for_CATEs_3, subset=grepl("Gun Control", for_CATEs_3$subtopic)),
     lm_robust(post_outcome_1 ~ treatment +
                scale(age, center = T) + scale(education, center = T) + scale(hhi, center = T) +
                scale(ideology, center = T) + scale(white, center = T) + 
                political_party, 
               for_CATEs_3, subset=grepl("Gun Control", for_CATEs_3$subtopic)),
     lm_robust(post_certain_scores ~ treatment, 
               for_CATEs_3, subset=grepl("Gun Control", for_CATEs_3$subtopic)),
     lm_robust(post_certain_scores ~ treatment +
                scale(age, center = T) + scale(education, center = T) + scale(hhi, center = T) +
                scale(ideology, center = T) + scale(white, center = T) + 
                political_party, 
               for_CATEs_3, subset=grepl("Gun Control", for_CATEs_3$subtopic))) %>%
 modelsummary(stars = T)
```


## Additional Study Details B.9 (Post-Treatment Attrition)

```{r}
library(tidyverse)
library(modelsummary)
source("scripts/clean_study4.R")
source("scripts/clean_study5.R")

study4_coded <- study4_coded %>% 
 mutate(pw2p_scale_na = is.na(pw2p_scale),
        pw2p_defend_na = is.na(pw2p_defend),
        treatment_f = factor(treatment, levels = c(0,1),
                             labels = c("Control", "High Valence")),
        pid_strength = if_else(pid7 == 1, 7,
                               if_else(pid7 == 2, 6,
                                       if_else(pid7 == 3, 5, pid7))) - 3)

study5_coded <- study5_coded %>% 
 mutate(pw2p_scale_na = is.na(pw2p_scale),
        pw2p_defend_na = is.na(pw2p_defend),
        treatment_f = factor(treatment, levels = c(0,1,2),
                             labels = c("Control", "Low Valence", "High Valence")),
        pid_strength = if_else(pid7 == 1, 7,
                               if_else(pid7 == 2, 6,
                                       if_else(pid7 == 3, 5, pid7))) - 3)

# Table B21
data.frame(
 Experiment = c(4, 4, 5, 5),
 Outcome = c("Extremity", "Defense", "Extremity", "Defense"),
 `Number missing` = c(sum(is.na(study4_coded$pw2p_scale)), 
                      sum(is.na(study4_coded$pw2p_defend)), 
                      sum(is.na(study5_coded$pw2p_scale)), 
                      sum(is.na(study5_coded$pw2p_defend))),
 `Percent missing` = c(mean(is.na(study4_coded$pw2p_scale))*100, 
                       mean(is.na(study4_coded$pw2p_defend))*100, 
                       mean(is.na(study5_coded$pw2p_scale))*100, 
                       mean(is.na(study5_coded$pw2p_defend))*100)
)

# Table B22
lm_study4_scale_na <- lm_robust(pw2p_scale_na ~ treatment_f, study4_coded)
lm_study4_defend_na <- lm_robust(pw2p_defend_na ~ treatment_f, study4_coded)
lm_study5_scale_na <- lm_robust(pw2p_scale_na ~ treatment_f, study5_coded)
lm_study5_defend_na <- lm_robust(pw2p_defend_na ~ treatment_f, study5_coded)
modelsummary(list(lm_study4_scale_na, lm_study4_defend_na, lm_study5_scale_na, lm_study5_defend_na),
             stars = TRUE)

# Table B23
lm_study4_scale_int <- lm_robust(pw2p_scale_na ~ treatment_f*likert + 
                           treatment_f*certainty + treatment_f*pid_strength, 
                          study4_coded)
lm_study4_defend_int <- lm_robust(pw2p_defend_na ~ treatment_f*likert + 
                            treatment_f*certainty + treatment_f*pid_strength, 
                           study4_coded)
lm_study5_scale_int <- lm_robust(pw2p_scale_na ~ treatment_f*likert + 
                                  treatment_f*certainty + treatment_f*pid_strength, 
                                 study5_coded)
lm_study5_defend_int <- lm_robust(pw2p_defend_na ~ treatment_f*likert + 
                                   treatment_f*certainty + treatment_f*pid_strength, 
                                  study5_coded)
modelsummary(list(lm_study4_scale_int, lm_study4_defend_int, lm_study5_scale_int, lm_study5_defend_int),
             stars = TRUE)
```

## Additional Study Details C.3 (Study Composition)

```{r}
s1 <- qualtRics::read_survey("data/study1.csv") %>%
 filter(Progress == 100)

s2 <- qualtRics::read_survey("data/study2_w2.csv") %>%
 filter(Progress == 100)

s3 <- qualtRics::read_survey("data/study3.csv") %>%
 filter(quality > 0 | is.na(quality)) %>%
 filter(is.na(generic_flag)) %>%
 filter(!grepl("farm", generic_flag)) %>% 
 filter(as.Date(StartDate, format='%m/%d/%Y')>="2022-12-12")

s4 <- qualtRics::read_survey("data/study4.csv") %>% 
 filter(moderation_failure == 0)

s5 <- qualtRics::read_survey("data/study5.csv") %>%
 filter(moderation_failure == 0 & DistributionChannel != "preview", !is.na(treatment))

demographics <- qualtRics::read_survey("data/cloudresearch_demographics.csv")

s1 %>%
 transmute(gender = as.numeric(gender == 2),
           age = 2022 - ifelse(birthyr > 2004 | birthyr < 1900, NA, birthyr),
           income,
           educ,
           ideology,
           dem = as.numeric(pid3==1),
           rep = as.numeric(pid3==2),
           ind = as.numeric(pid3==3))  %>%
 summarize_all(mean, na.rm = T) -> s1_demos

bind_cols(s1_demos,
       hispanic = mean(demographics$latino == 1),
       white = mean(demographics$latino == 0 & demographics$race == 1),
       black = mean(demographics$latino == 0 & demographics$race == 2),
       asian = mean(demographics$latino == 0 & demographics$race == 4)) -> s1_demos

s2 %>%
 transmute(gender = as.numeric(gender == 2),
           age = 2022 - ifelse(birthyr > 2004 | birthyr < 1900, NA, birthyr),
           income,
           educ,
           ideology,
           dem = as.numeric(pid3==1),
           rep = as.numeric(pid3==2),
           ind = as.numeric(pid3==3)) %>%
 summarize_all(mean, na.rm = T) -> s2_demos

bind_cols(s2_demos,
       hispanic = mean(demographics$latino == 1),
       white = mean(demographics$latino == 0 & demographics$race == 1),
       black = mean(demographics$latino == 0 & demographics$race == 2),
       asian = mean(demographics$latino == 0 & demographics$race == 4)) -> s2_demos


s3 %>%
 transmute(gender = as.numeric(gender == 2),
           age = age,
           income = if_else(hhi<0, NA, hhi),
           educ = if_else(education<0, NA, education),
           ideology,
           dem = as.numeric(political_party<3),
           rep = as.numeric(political_party>8),
           ind = as.numeric(political_party>=3 & political_party <=8),
           hispanic = as.numeric(hispanic != 1),
           white = as.numeric(ethnicity == 1 & hispanic != 1),
           black = as.numeric(ethnicity == 2 & hispanic != 1),
           asian = as.numeric(ethnicity >=4 & hispanic != 1 &
                               ethnicity <=14 & hispanic != 1))  %>%
 summarize_all(mean, na.rm = T) -> s3_demos

# Note: racial demographics in Experiments 4-5 were taken directly from the CloudResearch Connect platform

s4 %>%
 transmute(gender = as.numeric(gender...32 == 2),
           age = 2022 - ifelse(birthyr > 2004 | birthyr < 1900, NA, birthyr),
           income,
           educ,
           ideology...26,
           dem = as.numeric(pid3==1),
           rep = as.numeric(pid3==2),
           ind = as.numeric(pid3==3),
           hispanic = as.numeric(hispanic != 1),
           white = .74,
           hispanic = .12,
           black = .12,
           asian = .02)  %>%
 summarize_all(mean, na.rm = T) -> s4_demos

s5 %>%
 transmute(gender = as.numeric(gender...32 == 2),
           age = 2023-as.numeric(birthyr),
           income,
           educ,
           ideology = ideology...26,
           dem = as.numeric(pid3==1),
           rep = as.numeric(pid3==2),
           ind = as.numeric(pid3==3),
           white = .64,
           hispanic = .15,
           black = .15,
           asian = .05) %>%
 summarize_all(mean, na.rm = T) -> s5_demos

bind_rows(s1_demos,
          s2_demos,
          s3_demos,
          s4_demos,
          s5_demos) %>%
 round(digits = 2) %>% 
 t()
```
